# Import libraries
import h5py
import numpy as np
import pandas as pd
import librosa
import librosa.display

import matplotlib.pyplot as plt
from matplotlib import image

import os

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression, LassoCV
from sklearn.preprocessing import LabelBinarizer, StandardScaler, LabelEncoder
from sklearn.utils import class_weight
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, accuracy_score, precision_score, recall_score, f1_score, classification_report
from tensorflow.keras.preprocessing.image import load_img, img_to_array
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.callbacks import EarlyStopping

from tensorflow.keras import models, layers
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Flatten, Input
from tensorflow.keras.datasets import mnist, cifar10
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.applications.resnet import preprocess_input
from tensorflow.keras.applications import ResNet50
from tensorflow.keras.applications import imagenet_utils

# Evaluate hdf5 file. Check for key values
f = h5py.File('bird_spectrograms.hdf5', 'r')
for key in f.keys():
    print(key, f[key].shape)

amecro (128, 517, 66)
amerob (128, 517, 172)
bewwre (128, 517, 144)
bkcchi (128, 517, 45)
daejun (128, 517, 125)
houfin (128, 517, 84)
houspa (128, 517, 630)
norfli (128, 517, 37)
rewbla (128, 517, 187)
sonspa (128, 517, 263)
spotow (128, 517, 137)
whcspa (128, 517, 91)

# Example of Spectrogram for American Crow 
plt.figure(figsize=(10, 4))
librosa.display.specshow(f['amecro'][:,:,10], x_axis='time', y_axis='mel', cmap='gray_r')
plt.title('Mel Spectrogram: amecro clip 10')
plt.xlabel('Time')
plt.ylabel('Frequency (mel)')
plt.colorbar(format='%+2.0f dB')
plt.tight_layout()
plt.show()

# Add name identifier for species
bird_names = {
    "amecro": "American Crow",
    "amerob": "American Robin",
    "bewwre": "Bewick's Wren",
    "bkcchi": "Black-capped Chickadee",
    "daejun": "Dark-eyed Junco",
    "houfin": "House Finch",
    "houspa": "House Sparrow",
    "norfli": "Northern Flicker",
    "rewbla": "Red-winged Blackbird",
    "sonspa": "Song Sparrow",
    "spotow": "Spotted Towhee",
    "whcspa": "White-crowned Sparrow"
}

# Empty dictionary to hold spectrograms.
full_bird = {}

# Generate a dictionary for spectrograms. 
# This will make it easier to prep when doing a binary model.
for name in f.keys():
    # Transpose data from each species to N, 128, 517. 
    full_bird[name] = f[name][:].transpose((2,0,1))
    print(f"{bird_names[name]} ({name}):  {full_bird[name].shape}")

American Crow (amecro):  (66, 128, 517)
American Robin (amerob):  (172, 128, 517)
Bewick's Wren (bewwre):  (144, 128, 517)
Black-capped Chickadee (bkcchi):  (45, 128, 517)
Dark-eyed Junco (daejun):  (125, 128, 517)
House Finch (houfin):  (84, 128, 517)
House Sparrow (houspa):  (630, 128, 517)
Northern Flicker (norfli):  (37, 128, 517)
Red-winged Blackbird (rewbla):  (187, 128, 517)
Song Sparrow (sonspa):  (263, 128, 517)
Spotted Towhee (spotow):  (137, 128, 517)
White-crowned Sparrow (whcspa):  (91, 128, 517)

# Accuracy and loss graphs
def plot_training_history(history, title):
    plt.figure(figsize=(12, 5))

    # Accuracy plot
    plt.subplot(1, 2, 1)
    plt.plot(history.history['accuracy'], label='Train Accuracy')
    plt.plot(history.history['val_accuracy'], label='Val Accuracy')
    plt.title(f'{title} Accuracy Over Epochs')
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy')
    plt.legend(['Accuracy', 'Validation Accuracy'], loc='upper right')
    plt.grid(True)

    # Loss plot
    plt.subplot(1, 2, 2)
    plt.plot(history.history['loss'], label='Train Loss')
    plt.plot(history.history['val_loss'], label='Val Loss')
    plt.title(f'{title} Loss Over Epochs')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend(['Loss', 'Validation Loss'], loc='upper right')
    plt.grid(True)

    plt.tight_layout()
    plt.show()

def model_matrics(history, title, batch_size):
    print(f'\n{title}')
    
    final_train_acc = history.history['accuracy'][-1]
    final_val_acc = history.history['val_accuracy'][-1]
    final_train_loss = history.history['loss'][-1]
    final_val_loss = history.history['val_loss'][-1] 
    total_epochs = len(history.history['loss'])

    print(f"Epochs Trained: {total_epochs}")
    if batch_size is not None:
        print(f"Batch Size: {batch_size}")
    
    print(f"Final Training Accuracy: {final_train_acc:.3f}")
    print(f"Final Validation Accuracy: {final_val_acc:.3f}")
    print(f"Final Training Loss: {final_train_loss:.3f}")
    print(f"Final Validation Loss: {final_val_loss:.3f}")

# Label Binarizer to transform y values to 0 and 1. 
lb = LabelBinarizer()

# Function to generate data for binary model
def generate_binomial_data(species_list):

    # Empty lists to hold data
    X_list = []
    y_list = []
    
    # Loop through list containing species and append lists
    for species in species_list:
        specs = full_bird[species]
        X_list.append(specs)
        y_list.extend([species] * specs.shape[0])

    # Concatenate a list of spectrograms
    X = np.concatenate(X_list, axis=0)
    y = np.array(y_list)

    # Adding channel dimension
    X = X[..., np.newaxis]

    # Convert categorical labels for species to 0, 1. 
    y = lb.fit_transform(y).ravel()

    # Return prep data for training and testing. 
    # Stratifying on y, since I want equal representation in both training and test sets. 
    return train_test_split(X, y, test_size=0.33, random_state=13, stratify=y)

# Generate data for a binary model 
X_train, X_test, y_train, y_test = generate_binomial_data(['bewwre', 'spotow'])

# Evaluate contents of y_train and y_test
unique, counts = np.unique(y_train, return_counts=True)
print('y_train contents: ')
for label, count in zip(unique, counts):
    print(f"Class {label}: {count} samples")

unique, counts = np.unique(y_test, return_counts=True)
print('\ny_test contents: ')
for label, count in zip(unique, counts):
    print(f"Class {label}: {count} samples")

y_train contents: 
Class 0: 96 samples
Class 1: 92 samples

y_test contents: 
Class 0: 48 samples
Class 1: 45 samples

# Stop training if val_loss does not improve for 3 epochs and returns weights to best val_loss model
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

# Define model seq_model_simple(title, units, activation function, shape)
def seq_model_simple(title, units, act, shape):
    print(f'{title} (Dense Units: {units}, Activation Function: {act})')
    model = Sequential(
    [
    Dense(units=units, activation=act, input_shape=(shape,)),
    Dense(units=units, activation=act),
    Dense(units=1, activation='sigmoid')
    ])
    return model

# Define model seq_model_simple(title, units, activation function, shape, drop)
def seq_model_simple_drop(title, units, act, shape, drop):
    print(f'{title} (Dense Units: {units}, Activation Function: {act}, Dropout: {drop})')
    model = Sequential([
    Dense(units=units, activation=act, input_shape=(shape,)),
    Dropout(drop),
    Dense(units=units, activation=act),
    Dropout(drop),
    Dense(units=1, activation='sigmoid')])
    return model

# Compile model with seq_model_compile(title, defined model, optimization function, loss function, metric)
def seq_model_compile(title, model, opt, l, m):
    print(f'{title} (Optimizer Function: {opt}, Loss: {l}, Metric: {m})')
    model.compile(optimizer=opt
              , loss= l
              , metrics=[m])
    return model

# Test model simple_model(name, model, X_train, X_test, y_train, y_test, epoch, batch_size, early_stop)
def simple_model(title, model, xtrain, xtest, ytrain, ytest, e, b, stop):
    print(f'{title} (Epoch: {e}, Batch Size: {b})')
    history = model.fit(xtrain, ytrain, 
                    epochs=e, 
                    batch_size=b,
                    validation_data=(xtest, ytest),
                    callbacks=[stop],
                   verbose=0)
    name = f'{title} (Batch Size={b}, Epoch={e})'
    model_matrics(history, name, b)
    loss, acc = model.evaluate(xtest, ytest, verbose=0)
    print(f"Test Accuracy (from model.evaluate): {acc:.3f}")
    print(f"Trainable Params: {model.count_params()}")
    plot_training_history(history, title)
    return history

# MLP model expects a 1D vector, reshaping X_train and X_test
X_train_flat = X_train.reshape(X_train.shape[0] ,-1)
X_test_flat = X_test.reshape(X_test.shape[0] ,-1)

# Params to define model
input_dim = 128 * 517
u = [16, 32, 64]
act = 'relu'

# Params to compile model
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'

# Test model
title = 'Simple MLP'
b_sizes = [32, 64, 500]
epoch = 1000

results = []

# Loop to test different 
for u_size in u:
    print(f"\n=== Training {title} with Unit Size: {u_size} ===")
    for b_size in b_sizes:
        print(f"\n=== Training {title} with Batch Size: {b_size} ===")
        
        model = seq_model_simple(title, u_size, act, input_dim)
        model = seq_model_compile(title, model, opt, l, m)
        
        history = simple_model(title, 
                     model, 
                     X_train_flat, 
                     X_test_flat, 
                     y_train, 
                     y_test, 
                     epoch, 
                     b_size, 
                     stop=early_stop)
       
        y_pred_probs = model.predict(X_test_flat)
        y_pred = (y_pred_probs > 0.5).astype(int).flatten()

        cm = confusion_matrix(y_test, y_pred)
        tn, fp, fn, tp = cm.ravel()
        acc = accuracy_score(y_test, y_pred)
        precision = precision_score(y_test, y_pred, zero_division=0)
        recall = recall_score(y_test, y_pred, zero_division=0)
        f1 = f1_score(y_test, y_pred, zero_division=0)

        results.append({
            'Units': u_size,
            'Batch Size': b_size,
            'Epochs Train': len(history.history['loss']),
            'TP': tp,
            'TN': tn,
            'FP': fp,
            'FN': fn,
            'Accuracy': acc,
            'Precision': precision,
            'Recall': recall,
            'F1': f1
        })

results_df = pd.DataFrame(results)
display(results_df)

=== Training Simple MLP with Unit Size: 16 ===

=== Training Simple MLP with Batch Size: 32 ===
Simple MLP (Dense Units: 16, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 32)

/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Simple MLP (Batch Size=32, Epoch=1000)
Epochs Trained: 61
Batch Size: 32
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.505
Final Training Loss: 0.693
Final Validation Loss: 0.694
Test Accuracy (from model.evaluate): 0.505
Trainable Params: 1059121

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP with Batch Size: 64 ===
Simple MLP (Dense Units: 16, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 64)

/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Simple MLP (Batch Size=64, Epoch=1000)
Epochs Trained: 11
Batch Size: 64
Final Training Accuracy: 0.479
Final Validation Accuracy: 0.484
Final Training Loss: 23.222
Final Validation Loss: 24.531
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 1059121

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP with Batch Size: 500 ===
Simple MLP (Dense Units: 16, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 500)

/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Simple MLP (Batch Size=500, Epoch=1000)
Epochs Trained: 146
Batch Size: 500
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.516
Final Training Loss: 0.693
Final Validation Loss: 0.693
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 1059121

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP with Unit Size: 32 ===

=== Training Simple MLP with Batch Size: 32 ===
Simple MLP (Dense Units: 32, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 32)

# Scaled images to the [0, 1] range
X_train_scaled = X_train / 255
X_test_scaled = X_test / 255

# 1D vectors
X_train_flat = X_train_scaled.reshape(X_train_scaled.shape[0], -1)
X_test_flat = X_test_scaled.reshape(X_test_scaled.shape[0], -1)

# Params to define model
input_dim = 128 * 517
u = [16, 32, 64]
act = 'relu'

# Params to compile model
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'

# Test model
title = 'Simple MLP (Scaled)'
b_sizes = [32, 64, 500]
epoch = 1000

results = []

# Loop to test different 
for u_size in u:
    print(f"\n=== Training {title} with Unit Size: {u_size} ===")
    for b_size in b_sizes:
        print(f"\n=== Training {title} with Batch Size: {b_size} ===")
        
        model = seq_model_simple(title, u_size, act, input_dim)
        model = seq_model_compile(title, model, opt, l, m)
        
        history = simple_model(title, 
                     model, 
                     X_train_flat, 
                     X_test_flat, 
                     y_train, 
                     y_test, 
                     epoch, 
                     b_size, 
                     stop=early_stop)
       
        y_pred_probs = model.predict(X_test_flat)
        y_pred = (y_pred_probs > 0.5).astype(int).flatten()

        cm = confusion_matrix(y_test, y_pred)
        tn, fp, fn, tp = cm.ravel()
        acc = accuracy_score(y_test, y_pred)
        precision = precision_score(y_test, y_pred, zero_division=0)
        recall = recall_score(y_test, y_pred, zero_division=0)
        f1 = f1_score(y_test, y_pred, zero_division=0)

        results.append({
            'Units': u_size,
            'Batch Size': b_size,
            'Epochs Train': len(history.history['loss']),
            'TP': tp,
            'TN': tn,
            'FP': fp,
            'FN': fn,
            'Accuracy': acc,
            'Precision': precision,
            'Recall': recall,
            'F1': f1
        })

results_df = pd.DataFrame(results)
display(results_df)

=== Training Simple MLP (Scaled) with Unit Size: 16 ===

=== Training Simple MLP (Scaled) with Batch Size: 32 ===
Simple MLP (Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 32)

/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Simple MLP (Scaled) (Batch Size=32, Epoch=1000)
Epochs Trained: 10
Batch Size: 32
Final Training Accuracy: 0.441
Final Validation Accuracy: 0.516
Final Training Loss: 1.665
Final Validation Loss: 1.528
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 1059121

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Scaled) with Batch Size: 64 ===
Simple MLP (Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 64)

/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Simple MLP (Scaled) (Batch Size=64, Epoch=1000)
Epochs Trained: 19
Batch Size: 64
Final Training Accuracy: 0.585
Final Validation Accuracy: 0.516
Final Training Loss: 0.687
Final Validation Loss: 0.697
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 1059121

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Scaled) with Batch Size: 500 ===
Simple MLP (Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 500)

/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Simple MLP (Scaled) (Batch Size=500, Epoch=1000)
Epochs Trained: 15
Batch Size: 500
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.484
Final Training Loss: 1.861
Final Validation Loss: 5.392
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 1059121

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Scaled) with Unit Size: 32 ===

=== Training Simple MLP (Scaled) with Batch Size: 32 ===
Simple MLP (Scaled) (Dense Units: 32, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 32)

# 1D vectors
X_train_flat = X_train.reshape(X_train.shape[0] ,-1)
X_test_flat = X_test.reshape(X_test.shape[0] ,-1)

mean = np.mean(X_train_flat)
std = np.std(X_train_flat)

X_train_flat_scaled = (X_train_flat - mean) / std
X_test_flat_scaled = (X_test_flat - mean) / std


# Params to define model
input_dim = 128 * 517
u = [16, 32, 64]
act = 'relu'

# Params to compile model
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'

# Test model
title = 'Simple MLP (Z Scaled)'
b_sizes = [32, 64, 500]
epoch = 1000

results = []

# Loop to test different 
for u_size in u:
    print(f"\n=== Training {title} with Unit Size: {u_size} ===")
    for b_size in b_sizes:
        print(f"\n=== Training {title} with Batch Size: {b_size} ===")
        
        model = seq_model_simple(title, u_size, act, input_dim)
        model = seq_model_compile(title, model, opt, l, m)
        
        history = simple_model(title, 
                     model, 
                     X_train_flat, 
                     X_test_flat, 
                     y_train, 
                     y_test, 
                     epoch, 
                     b_size, 
                     stop=early_stop)
       
        y_pred_probs = model.predict(X_test_flat)
        y_pred = (y_pred_probs > 0.5).astype(int).flatten()

        cm = confusion_matrix(y_test, y_pred)
        tn, fp, fn, tp = cm.ravel()
        acc = accuracy_score(y_test, y_pred)
        precision = precision_score(y_test, y_pred, zero_division=0)
        recall = recall_score(y_test, y_pred, zero_division=0)
        f1 = f1_score(y_test, y_pred, zero_division=0)

        results.append({
            'Units': u_size,
            'Batch Size': b_size,
            'Epochs Train': len(history.history['loss']),
            'TP': tp,
            'TN': tn,
            'FP': fp,
            'FN': fn,
            'Accuracy': acc,
            'Precision': precision,
            'Recall': recall,
            'F1': f1
        })

results_df = pd.DataFrame(results)
display(results_df)

=== Training Simple MLP (Z Scaled) with Unit Size: 16 ===

=== Training Simple MLP (Z Scaled) with Batch Size: 32 ===
Simple MLP (Z Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 32)

/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Simple MLP (Z Scaled) (Batch Size=32, Epoch=1000)
Epochs Trained: 11
Batch Size: 32
Final Training Accuracy: 0.516
Final Validation Accuracy: 0.516
Final Training Loss: 406.022
Final Validation Loss: 766.301
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 1059121

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Z Scaled) with Batch Size: 64 ===
Simple MLP (Z Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 64)

/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Simple MLP (Z Scaled) (Batch Size=64, Epoch=1000)
Epochs Trained: 6
Batch Size: 64
Final Training Accuracy: 0.489
Final Validation Accuracy: 0.516
Final Training Loss: 8.166
Final Validation Loss: 7.217
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 1059121

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Z Scaled) with Batch Size: 500 ===
Simple MLP (Z Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 500)

/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Simple MLP (Z Scaled) (Batch Size=500, Epoch=1000)
Epochs Trained: 9
Batch Size: 500
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.484
Final Training Loss: 148.349
Final Validation Loss: 378.533
Test Accuracy (from model.evaluate): 0.462
Trainable Params: 1059121

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step 

=== Training Simple MLP (Z Scaled) with Unit Size: 32 ===

=== Training Simple MLP (Z Scaled) with Batch Size: 32 ===
Simple MLP (Z Scaled) (Dense Units: 32, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 32)

# Build CNN model
def build_cnn_model(name, layers, channels, k_size, pool, act, drop, dense, u):
    print(f'{name} with {layers} layer(s), {channels} channels, {k_size} kernel size, {pool} pool size, {act} activation, {drop} dropout, {dense} hidden dense layer(s)')
    
    model = Sequential()
    model.add(Input(shape=(128, 517, 1)))
    model.add(Conv2D(filters=channels, kernel_size=k_size, padding='same', activation=act))
    model.add(MaxPooling2D(pool_size=pool))

    for i in range(1, layers):
        model.add(Conv2D(filters=channels * (2**i), kernel_size=k_size, padding='same', activation=act))
        model.add(MaxPooling2D(pool_size=pool))

    model.add(Flatten())
    model.add(Dropout(drop))
    
    units = u
    for j in range(dense):
        units = int(units)
        model.add(Dense(units=units, activation=act))
        if drop > 0:
            model.add(Dropout(drop)) 
        units = units/2

    model.add(Dense(1, activation='sigmoid'))
    return model

# Compile function
def seq_model_compile(title, model, opt, l, m):
    print(f'{title} Compiled (Optimizer: {opt}, Loss: {l}, Metric: {m})')
    model.compile(optimizer=opt, loss=l, metrics=[m])
    return model

# Train and evaluate
def cnn_model(title, model, xtrain, xtest, ytrain, ytest, e, b, stop):
    print(f'{title} Tested (Optimizer: {opt}, Loss: {l}, Metric: {m})')
    history = model.fit(xtrain, ytrain, 
                        epochs=e, 
                        batch_size=b,
                        validation_data=(xtest, ytest),
                        callbacks=[stop],
                        verbose=0)
    name = f'{title} (Batch Size={b}, Epoch={e})'
    model_matrics(history, name, b)
    loss, acc = model.evaluate(xtest, ytest, verbose=0)
    print(f"Test Accuracy: {acc:.3f}")
    print(f"Trainable Params: {model.count_params()}")
    plot_training_history(history, title)
    return history

def cnn_metrics(model, y_test, X_test_scaled, layers, channels, lb):
    y_pred_probs = model.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    label_map = f"{lb.classes_[0]}=0, {lb.classes_[1]}=1"

    results_df = pd.DataFrame([{
        'Label Mapping': label_map,
        'Layers': layers,
        'Channels': channels,
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    }])
    
    display(results_df)

# Generate data for a binary model
X_train, X_test, y_train, y_test = generate_binomial_data(['bewwre', 'spotow'])
X_train_scaled = X_train / 255.0
X_test_scaled = X_test / 255.0

# Params
layer_list = [1]
channels = 16
k_size = (3, 3)
pool = (2, 2)
act = 'relu'
dropout = 0.5
dense_layers = 0
units = 128
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for layers in layer_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Scaled Data with 1 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 12
Batch Size: 32
Final Training Accuracy: 0.697
Final Validation Accuracy: 0.516
Final Training Loss: 0.570
Final Validation Loss: 0.822
Test Accuracy: 0.602
Trainable Params: 264353

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 25ms/step

mean = np.mean(X_train)
std = np.std(X_train)

X_train_scaled = (X_train - mean) / std
X_test_scaled = (X_test - mean) / std

# Params
layer_list = [1]
channels = 16
k_size = (3, 3)
pool = (2, 2)
act = 'relu'
dropout = 0.5
dense_layers = 0
units = 128
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for layers in layer_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 1 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 18
Batch Size: 32
Final Training Accuracy: 0.676
Final Validation Accuracy: 0.559
Final Training Loss: 1.025
Final Validation Loss: 0.838
Test Accuracy: 0.645
Trainable Params: 264353

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 25ms/step

layer_list = [1, 2, 3]
channels = 16
k_size = (3, 3)
pool = (2, 2)
act = 'relu'
dropout = 0.5
dense_layers = 0
units = 128
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for layers in layer_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 1 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 8
Batch Size: 32
Final Training Accuracy: 0.686
Final Validation Accuracy: 0.484
Final Training Loss: 1.313
Final Validation Loss: 4.232
Test Accuracy: 0.581
Trainable Params: 264353

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 25ms/step
CNN Z-Scaled Data with 2 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 12
Batch Size: 32
Final Training Accuracy: 0.936
Final Validation Accuracy: 0.570
Final Training Loss: 0.231
Final Validation Loss: 0.675
Test Accuracy: 0.634
Trainable Params: 136897

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 37ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 15
Batch Size: 32
Final Training Accuracy: 0.851
Final Validation Accuracy: 0.613
Final Training Loss: 0.330
Final Validation Loss: 0.818
Test Accuracy: 0.645
Trainable Params: 88833

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 44ms/step

layers = 3
channels = 16
k_size = (3, 3)
pool_list = [(2, 2), (3,3), (4,4), (5,5)]
act = 'relu'
dropout = 0.5
dense_layers = 0
units = 128
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for pool in pool_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 14
Batch Size: 32
Final Training Accuracy: 0.915
Final Validation Accuracy: 0.527
Final Training Loss: 0.316
Final Validation Loss: 1.285
Test Accuracy: 0.624
Trainable Params: 88833

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 49ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 7
Batch Size: 32
Final Training Accuracy: 0.739
Final Validation Accuracy: 0.667
Final Training Loss: 0.566
Final Validation Loss: 0.614
Test Accuracy: 0.720
Trainable Params: 28161

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 34ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (4, 4) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 13
Batch Size: 32
Final Training Accuracy: 0.739
Final Validation Accuracy: 0.656
Final Training Loss: 0.557
Final Validation Loss: 0.619
Test Accuracy: 0.688
Trainable Params: 24321

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 32ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (5, 5) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 14
Batch Size: 32
Final Training Accuracy: 0.649
Final Validation Accuracy: 0.591
Final Training Loss: 0.609
Final Validation Loss: 0.688
Test Accuracy: 0.613
Trainable Params: 23553

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 29ms/step

layers = 3
channels = 16
k_size = (3, 3)
pool = (3,3)
act = 'relu'
dropout = 0.5
dense_layers = 1
units_list = [512, 256, 128]
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for units in units_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 14
Batch Size: 32
Final Training Accuracy: 0.803
Final Validation Accuracy: 0.538
Final Training Loss: 0.454
Final Validation Loss: 0.810
Test Accuracy: 0.645
Trainable Params: 2514689

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 38ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 15
Batch Size: 32
Final Training Accuracy: 0.755
Final Validation Accuracy: 0.591
Final Training Loss: 0.497
Final Validation Loss: 0.641
Test Accuracy: 0.699
Trainable Params: 1268993

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 38ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 9
Batch Size: 32
Final Training Accuracy: 0.665
Final Validation Accuracy: 0.570
Final Training Loss: 0.559
Final Validation Loss: 0.628
Test Accuracy: 0.634
Trainable Params: 646145

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 36ms/step

layers = 3
channels = 16
k_size_list = [(3, 3), (4,4), (5,5)]
pool = (3,3)
act = 'relu'
dropout = 0.5
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for k_size in k_size_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 16
Batch Size: 32
Final Training Accuracy: 0.819
Final Validation Accuracy: 0.570
Final Training Loss: 0.444
Final Validation Loss: 0.675
Test Accuracy: 0.667
Trainable Params: 1268993

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 37ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (4, 4) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 11
Batch Size: 32
Final Training Accuracy: 0.676
Final Validation Accuracy: 0.527
Final Training Loss: 0.557
Final Validation Loss: 0.667
Test Accuracy: 0.645
Trainable Params: 1287025

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 43ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 15
Batch Size: 32
Final Training Accuracy: 0.782
Final Validation Accuracy: 0.656
Final Training Loss: 0.518
Final Validation Loss: 0.632
Test Accuracy: 0.677
Trainable Params: 1310209

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 48ms/step

layers = 3
channels = 16
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout = 0.5
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch_list = [16, 32, 64]
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for batch in batch_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=16, Epoch=100)
Epochs Trained: 11
Batch Size: 16
Final Training Accuracy: 0.745
Final Validation Accuracy: 0.591
Final Training Loss: 0.496
Final Validation Loss: 0.671
Test Accuracy: 0.613
Trainable Params: 1310209

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 50ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 14
Batch Size: 32
Final Training Accuracy: 0.771
Final Validation Accuracy: 0.602
Final Training Loss: 0.545
Final Validation Loss: 0.636
Test Accuracy: 0.613
Trainable Params: 1310209

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 49ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 22
Batch Size: 64
Final Training Accuracy: 0.745
Final Validation Accuracy: 0.548
Final Training Loss: 0.498
Final Validation Loss: 0.675
Test Accuracy: 0.688
Trainable Params: 1310209

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 51ms/step

layers = 3
channels_list = [16, 32]
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout = 0.5
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch_list = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for channels in channels_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 20
Batch Size: 64
Final Training Accuracy: 0.681
Final Validation Accuracy: 0.591
Final Training Loss: 0.586
Final Validation Loss: 0.704
Test Accuracy: 0.634
Trainable Params: 1310209

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 47ms/step
CNN Z-Scaled Data with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 20
Batch Size: 64
Final Training Accuracy: 0.617
Final Validation Accuracy: 0.602
Final Training Loss: 0.655
Final Validation Loss: 0.644
Test Accuracy: 0.613
Trainable Params: 2747905

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 88ms/step

layers = 3
channels = 16
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout = 0.5
dense_layers_list = [1, 2, 3]
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for dense_layers in dense_layers_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 26
Batch Size: 64
Final Training Accuracy: 0.718
Final Validation Accuracy: 0.505
Final Training Loss: 0.564
Final Validation Loss: 1.049
Test Accuracy: 0.634
Trainable Params: 1310209

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 54ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 2 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 20
Batch Size: 64
Final Training Accuracy: 0.707
Final Validation Accuracy: 0.667
Final Training Loss: 0.590
Final Validation Loss: 0.607
Test Accuracy: 0.656
Trainable Params: 1342977

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 48ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 3 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 18
Batch Size: 64
Final Training Accuracy: 0.601
Final Validation Accuracy: 0.538
Final Training Loss: 0.646
Final Validation Loss: 0.712
Test Accuracy: 0.602
Trainable Params: 1351169

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 50ms/step

layers = 3
channels = 16
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout_list = [0.5, 0.4, 0.3]
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

cnn_results = []

for dropout in dropout_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 25
Batch Size: 64
Final Training Accuracy: 0.777
Final Validation Accuracy: 0.624
Final Training Loss: 0.511
Final Validation Loss: 0.632
Test Accuracy: 0.624
Trainable Params: 1310209

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 42ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.4 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 21
Batch Size: 64
Final Training Accuracy: 0.734
Final Validation Accuracy: 0.581
Final Training Loss: 0.583
Final Validation Loss: 0.661
Test Accuracy: 0.591
Trainable Params: 1310209

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 46ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.3 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 25
Batch Size: 64
Final Training Accuracy: 0.771
Final Validation Accuracy: 0.613
Final Training Loss: 0.503
Final Validation Loss: 0.626
Test Accuracy: 0.699
Trainable Params: 1310209

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 48ms/step

layers = 3
channels = 16
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout = 0.3
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

cnn_results = []


model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
model_cnn = seq_model_compile(title, model, opt, l, m)
history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)
y_pred_probs = model_cnn.predict(X_test_scaled)
y_pred = (y_pred_probs > 0.5).astype(int).flatten()
cm = confusion_matrix(y_test, y_pred)
tn, fp, fn, tp = cm.ravel()
acc = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred, zero_division=0)
recall = recall_score(y_test, y_pred, zero_division=0)
f1 = f1_score(y_test, y_pred, zero_division=0)
cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.3 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 16
Batch Size: 64
Final Training Accuracy: 0.723
Final Validation Accuracy: 0.634
Final Training Loss: 0.559
Final Validation Loss: 0.665
Test Accuracy: 0.613
Trainable Params: 1310209

3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 52ms/step

X_train, X_test, y_train, y_test = generate_binomial_data(['amecro', 'houspa'])
mean = np.mean(X_train)
std = np.std(X_train)

X_train_scaled = (X_train - mean) / std
X_test_scaled = (X_test - mean) / std

layers = 3
channels = 16
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout = 0.3
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

cnn_results = []


model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
model_cnn = seq_model_compile(title, model, opt, l, m)
history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)
y_pred_probs = model_cnn.predict(X_test_scaled)
y_pred = (y_pred_probs > 0.5).astype(int).flatten()
cm = confusion_matrix(y_test, y_pred)
tn, fp, fn, tp = cm.ravel()
acc = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred, zero_division=0)
recall = recall_score(y_test, y_pred, zero_division=0)
f1 = f1_score(y_test, y_pred, zero_division=0)
cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)

CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.3 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 19
Batch Size: 64
Final Training Accuracy: 0.989
Final Validation Accuracy: 0.965
Final Training Loss: 0.025
Final Validation Loss: 0.070
Test Accuracy: 0.978
Trainable Params: 1310209

8/8 ━━━━━━━━━━━━━━━━━━━━ 0s 36ms/step

# Lock Model
model_cnn.save('final_bird_model.keras')

def generate_multiclass_data(full_bird):
    X_list = []
    y_list = []

    for species in full_bird.keys():
        specs = full_bird[species]
        X_list.append(specs)
        y_list.extend([species] * specs.shape[0])

    X = np.concatenate(X_list, axis=0)
    y = np.array(y_list)
    X = X[..., np.newaxis]

    label = LabelEncoder()
    y_encoded = label.fit_transform(y)
    y_cat = to_categorical(y_encoded)


    X_flat = X.reshape(X.shape[0], -1)
    mean = X_flat.mean()
    std = X_flat.std()
    X = (X - mean) / std

    return train_test_split(X, y_cat, test_size=0.33, random_state=13, stratify=y_encoded), label

# Build CNN model
def build_multi_cnn_model(name, layers, channels, k_size, pool, act, drop, dense, u):
    print(f'{name} with {layers} layer(s), {channels} channels, {k_size} kernel size, {pool} pool, {act} activation, {drop} dropout, {dense} dense layer(s)')
    
    model = Sequential()
    model.add(Input(shape=(128, 517, 1)))
    model.add(Conv2D(filters=channels, kernel_size=k_size, padding='same', activation=act))
    model.add(MaxPooling2D(pool_size=pool))

    for i in range(1, layers):
        model.add(Conv2D(filters=channels * (2**i), kernel_size=k_size, padding='same', activation=act))
        model.add(MaxPooling2D(pool_size=pool))

    model.add(Flatten())
    model.add(Dropout(drop))

    units = u
    for j in range(dense):
        units = int(units)
        model.add(Dense(units=units, activation=act))
        if drop > 0:
            model.add(Dropout(drop)) 
        units = units/2  

    model.add(Dense(12, activation='softmax'))
    return model

# Compile Model
def seq_model_compile(title, model, opt, l, m):
    print(f'{title} (Optimizer: {opt}, Loss: {l}, Metric: {m})')
    model.compile(optimizer=opt, loss=l, metrics=[m])
    return model

# Train and evaluate
def cnn_model(title, model, xtrain, xtest, ytrain, ytest, e, b, stop):
    print(f'{title} (Batch Size={b}, Epoch={e})')
    history = model.fit(xtrain, ytrain, 
                        epochs=e, 
                        batch_size=b,
                        validation_data=(xtest, ytest),
                        callbacks=[stop],
                        verbose=0)
    plot_training_history(history, title)
    return history

def cnn_multi_metrics(model, X_test, y_test_cat, layers, channels, label_encoder):
    y_pred_probs = model.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test_cat, axis=1)

    acc = accuracy_score(y_true, y_pred)
    report = classification_report(y_true, y_pred, target_names=label_encoder.classes_, output_dict=True, zero_division=0)

    print(f"\nAccuracy: {acc:.3f}")
    print("\nClassification Report:")
    print(classification_report(y_true, y_pred, target_names=label_encoder.classes_, zero_division=0))

# Load and prepare data
(X_train, X_test, y_train, y_test), label_encoder = generate_multiclass_data(full_bird)

channels = 32
layers_list = [1,2,3]
k_size = (5, 5)
pool = (3, 3)
act = 'relu'
drop = 0.3
dense_layers = 1
units = 256
opt = 'rmsprop'
loss = 'categorical_crossentropy'
metric = 'accuracy'
title = 'CNN Multi-Class'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)


results = []

for layers in layers_list:
    model = build_multi_cnn_model(title, layers, channels, k_size, pool, act, drop, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, loss, metric)
    history = cnn_model(title, model_cnn, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test, axis=1)

    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
    recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
    precision = precision_score(y_true, y_pred, average='macro', zero_division=0)

    results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })

results_df = pd.DataFrame(results)
display(results_df)

CNN Multi-Class with 1 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step
CNN Multi-Class with 2 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 1s 65ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 79ms/step

channels = 32
layers = 3
k_size = (5, 5)
pool = (3, 3)
act = 'relu'
drop = 0.3
dense_layers_list = [1, 2]
units = 256
opt = 'rmsprop'
loss = 'categorical_crossentropy'
metric = 'accuracy'
title = 'CNN Multi-Class'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)


results = []

for dense_layers in dense_layers_list:
    model = build_multi_cnn_model(title, layers, channels, k_size, pool, act, drop, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, loss, metric)
    history = cnn_model(title, model_cnn, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test, axis=1)

    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
    recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
    precision = precision_score(y_true, y_pred, average='macro', zero_division=0)

    results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })

results_df = pd.DataFrame(results)
display(results_df)

CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 78ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 2 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 79ms/step

channels = 32
layers = 3
k_size = (5, 5)
pool = (3, 3)
act = 'relu'
drop_list = [0.2, 0.3, 0.4]
dense_layers = 1
units = 256
opt = 'rmsprop'
loss = 'categorical_crossentropy'
metric = 'accuracy'
title = 'CNN Multi-Class'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)


results = []

for drop in drop_list:
    model = build_multi_cnn_model(title, layers, channels, k_size, pool, act, drop, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, loss, metric)
    history = cnn_model(title, model_cnn, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test, axis=1)

    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
    recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
    precision = precision_score(y_true, y_pred, average='macro', zero_division=0)

    results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': drop,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })

results_df = pd.DataFrame(results)
display(results_df)

CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.2 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 82ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 80ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 80ms/step

channels = 32
layers = 3
k_size_list = [(3, 3), (5, 5), (7, 7)]
pool = (3, 3)
act = 'relu'
drop = 0.4
dense_layers = 1
units = 256
opt = 'rmsprop'
loss = 'categorical_crossentropy'
metric = 'accuracy'
title = 'CNN Multi-Class'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)


results = []

for k_size in k_size_list:
    model = build_multi_cnn_model(title, layers, channels, k_size, pool, act, drop, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, loss, metric)
    history = cnn_model(title, model_cnn, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test, axis=1)

    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
    recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
    precision = precision_score(y_true, y_pred, average='macro', zero_division=0)

    results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })

results_df = pd.DataFrame(results)
display(results_df)

CNN Multi-Class with 3 layer(s), 32 channels, (3, 3) kernel size, (3, 3) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 1s 48ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 87ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (7, 7) kernel size, (3, 3) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 3s 154ms/step

channels = 32
layers = 3
k_size = (7, 7)
pool_list = [(2,2),(3, 3),(4,4)]
act = 'relu'
drop = 0.4
dense_layers = 1
units = 256
opt = 'rmsprop'
loss = 'categorical_crossentropy'
metric = 'accuracy'
title = 'CNN Multi-Class'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)


results = []

for pool in pool_list:
    model = build_multi_cnn_model(title, layers, channels, k_size, pool, act, drop, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, loss, metric)
    history = cnn_model(title, model_cnn, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test, axis=1)

    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
    recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
    precision = precision_score(y_true, y_pred, average='macro', zero_division=0)

    results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': drop,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })

results_df = pd.DataFrame(results)
display(results_df)

CNN Multi-Class with 3 layer(s), 32 channels, (7, 7) kernel size, (2, 2) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 8s 355ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (7, 7) kernel size, (3, 3) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 3s 135ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (7, 7) kernel size, (4, 4) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 83ms/step

# Final hyperparameters
layers = 3
channels = 32
k_size = (7, 7)
pool = (4, 4)
dropout = 0.4
dense_layers = 1
units = 256
batch = 64
epoch = 100
title = "CNN Multi-Class Final"
early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

# Build and compile
model = build_multi_cnn_model(title, layers, channels, k_size, pool, 'relu', dropout, dense_layers, units)
model = seq_model_compile(title, model, 'rmsprop', 'categorical_crossentropy', 'accuracy')
history = cnn_model(title, model, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

# Predict
y_pred_probs = model.predict(X_test)
y_pred = np.argmax(y_pred_probs, axis=1)
y_true = np.argmax(y_test, axis=1)

# Confusion matrix
cm = confusion_matrix(y_true, y_pred)
cm_df = pd.DataFrame(cm,
                     index=label_encoder.classes_,  
                     columns=label_encoder.classes_)  

print("Confusion Matrix (Table Format):")
display(cm_df)

CNN Multi-Class Final with 3 layer(s), 32 channels, (7, 7) kernel size, (4, 4) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class Final (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class Final (Batch Size=64, Epoch=100)

21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 93ms/step
Confusion Matrix (Table Format):

results = []
acc = accuracy_score(y_true, y_pred)
f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
precision = precision_score(y_true, y_pred, average='macro', zero_division=0)
results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': drop,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })
results_df = pd.DataFrame(results)
display(results_df)

model.save("cnn_bird_classifier.keras")

# Path
test_dir = '/Users/markdaza/Library/CloudStorage/OneDrive-SeattleUniversity/DATA 5322/Homework3/test_birds'
test_files = ['test1.mp3', 'test2.mp3', 'test3.mp3']

# Array for spectrograms
test_specs = np.zeros((128, 517, len(test_files)))

# Generate spectrograms
for i, fname in enumerate(test_files):
    full_path = os.path.join(test_dir, fname)
    y, sr = librosa.load(full_path, duration=3)
    
    # Compute mel spectrogram
    spec = librosa.feature.melspectrogram(y=y, sr=sr, hop_length=128, win_length=512, n_mels=128)
    spec_db = librosa.power_to_db(spec, ref=np.max)
    
    # Standardize w
    if spec_db.shape[1] < 517:
        pad_width = 517 - spec_db.shape[1]
        spec_db = np.pad(spec_db, ((0, 0), (0, pad_width)), mode='constant')
    else:
        spec_db = spec_db[:, :517]
    
    test_specs[:, :, i] = spec_db

print("All spectrograms processed. Shape:", test_specs.shape)

All spectrograms processed. Shape: (128, 517, 3)

plt.figure(figsize=(10, 4))
librosa.display.specshow(test_specs[:, :, 0], x_axis='time', y_axis='mel', sr=sr, cmap='gray_r')
plt.colorbar(format='%+2.0f dB')
plt.title("Mel Spectrogram: test1.mp3")
plt.tight_layout()
plt.show()

plt.figure(figsize=(10, 4))
librosa.display.specshow(test_specs[:, :, 1], x_axis='time', y_axis='mel', sr=sr, cmap='gray_r')
plt.colorbar(format='%+2.0f dB')
plt.title("Mel Spectrogram: test2.mp3")
plt.tight_layout()
plt.show()

plt.figure(figsize=(10, 4))
librosa.display.specshow(test_specs[:, :, 2], x_axis='time', y_axis='mel', sr=sr, cmap='gray_r')
plt.colorbar(format='%+2.0f dB')
plt.title("Mel Spectrogram: test3.mp3")
plt.tight_layout()
plt.show()

# Extract mean and std from training set
mean = X_train.mean()
std = X_train.std()

# Apply Z-scaling to spectrograms
test_specs_scaled = (test_specs - mean) / std
test_specs_scaled = np.transpose(test_specs_scaled, (2, 0, 1)) 
test_specs_scaled = test_specs_scaled[..., np.newaxis]  

# Load Model
model_multi = load_model('cnn_bird_classifier.keras')

# Predict 
y_pred_probs = model_multi.predict(test_specs_scaled)  

# Get predictions for each spectrogram
top3_indices = np.argsort(y_pred_probs, axis=1)[:, -3:][:, ::-1] 
top3_probs = np.sort(y_pred_probs, axis=1)[:, -3:][:, ::-1]

# Decode labels of predictions to bird species
top3_labels = label_encoder.inverse_transform(top3_indices.flatten()).reshape(3, 3)

# Top predictions
for i, (labels, probs) in enumerate(zip(top3_labels, top3_probs)):
    print(f"\ntest{i+1}.mp3 - Top 3 Predictions:")
    for j in range(3):
        print(f"  {j+1}. {labels[j]} (prob: {probs[j]:.3f})")

1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 37ms/step

test1.mp3 - Top 3 Predictions:
  1. houspa (prob: 1.000)
  2. rewbla (prob: 0.000)
  3. whcspa (prob: 0.000)

test2.mp3 - Top 3 Predictions:
  1. houspa (prob: 0.904)
  2. spotow (prob: 0.096)
  3. rewbla (prob: 0.000)

test3.mp3 - Top 3 Predictions:
  1. houspa (prob: 1.000)
  2. spotow (prob: 0.000)
  3. whcspa (prob: 0.000)

Observation:¶

CNN Hyperparameter Optimization¶

Layer Optimization¶

Observation: 3 layers gave us good accuracy and Precision, but lower F1 scores. We will proceed with three layers for now¶

Pool Optimization¶

Observation: Higher performance of pool size (3,3)¶

Dense layer Units Optmization¶

Observation: 256 units in the hidden dense layer gave us higher accuracy, better recall and F1 score.¶

K Size Optimization¶

Observation: (5,5) performed best in terms of accuracy, recall and F1 score¶

Batch Optimization¶

Observation: Batch size of 64 provided better accuracy, recall, and F1 scores.¶

Channel Optimization¶

Observation: Keep the Number of chanels the same.¶

Dense Layer tuning¶

Observation: Dense layers should be 1 at 256 units¶

Dropout Optimization¶

Observation: Dropout at 0.3 improved accuracy, recall and F1 score.¶

Final Model¶

Test on different set of birds¶

Multi Class Prediction¶

Number of layers optimizations¶

Number of Dense Layers Optimization¶

Observations: 1 dense layer is enough for the model¶

Dropout Optimization¶

Observations: Setting dropout to 0.4 improved accuracy, F1 and precision.¶

K Size Optimization¶

Observations: (7,7) kernel size performed the best with higher accuracy, F1 and Recall¶

Pool Optimization¶

Observations: Pooling (4,4) showed higher accuracy, F1, precission¶

Final model¶

Generate Spectograms for test files¶

	Units	Batch Size	Epochs Train	TP	TN	FP	FN	Accuracy	Precision	Recall	F1
0	16	32	61	0	47	1	45	0.505376	0.000000	0.0	0.000000
1	16	64	11	45	0	48	0	0.483871	0.483871	1.0	0.652174
2	16	500	146	0	48	0	45	0.516129	0.000000	0.0	0.000000
3	32	32	8	0	48	0	45	0.516129	0.000000	0.0	0.000000
4	32	64	6	45	0	48	0	0.483871	0.483871	1.0	0.652174
5	32	500	5	0	48	0	45	0.516129	0.000000	0.0	0.000000
6	64	32	24	0	48	0	45	0.516129	0.000000	0.0	0.000000
7	64	64	7	9	45	3	36	0.580645	0.750000	0.2	0.315789
8	64	500	10	45	0	48	0	0.483871	0.483871	1.0	0.652174

	Units	Batch Size	Epochs Train	TP	TN	FP	FN	Accuracy	Precision	Recall	F1
0	16	32	10	1	47	1	44	0.516129	0.500000	0.022222	0.042553
1	16	64	19	0	48	0	45	0.516129	0.000000	0.000000	0.000000
2	16	500	15	0	48	0	45	0.516129	0.000000	0.000000	0.000000
3	32	32	5	1	48	0	44	0.526882	1.000000	0.022222	0.043478
4	32	64	67	0	48	0	45	0.516129	0.000000	0.000000	0.000000
5	32	500	6	10	43	5	35	0.569892	0.666667	0.222222	0.333333
6	64	32	5	0	48	0	45	0.516129	0.000000	0.000000	0.000000
7	64	64	9	45	0	48	0	0.483871	0.483871	1.000000	0.652174
8	64	500	6	45	0	48	0	0.483871	0.483871	1.000000	0.652174

	Title	Layers	Channels	Kernel	Pool	Dropout	Dense Units	Batch Size	Epochs Trained	Trainable Params	TP	TN	FP	FN	Accuracy	Precision	Recall	F1
0	CNN Z-Scaled Data	1	16	(3, 3)	(2, 2)	0.5	128	32	8	264353	26	28	20	19	0.580645	0.565217	0.577778	0.571429
1	CNN Z-Scaled Data	2	16	(3, 3)	(2, 2)	0.5	128	32	12	136897	25	34	14	20	0.634409	0.641026	0.555556	0.595238
2	CNN Z-Scaled Data	3	16	(3, 3)	(2, 2)	0.5	128	32	15	88833	19	41	7	26	0.645161	0.730769	0.422222	0.535211

	Title	Layers	Channels	Kernel	Pool	Dropout	Dense Layers	Dense Units	Batch Size	Epochs Trained	Trainable Params	TP	TN	FP	FN	Accuracy	Precision	Recall	F1
0	CNN Z-Scaled Data	3	16	(5, 5)	(3, 3)	0.5	1	256	16	11	1310209	18	39	9	27	0.612903	0.666667	0.4	0.500000
1	CNN Z-Scaled Data	3	16	(5, 5)	(3, 3)	0.5	1	256	32	14	1310209	18	39	9	27	0.612903	0.666667	0.4	0.500000
2	CNN Z-Scaled Data	3	16	(5, 5)	(3, 3)	0.5	1	256	64	22	1310209	36	28	20	9	0.688172	0.642857	0.8	0.712871

	Title	Layers	Channels	Kernel	Pool	Dropout	Dense Layers	Dense Units	Batch Size	Epochs Trained	Trainable Params	Accuracy	Macro F1	Macro Precision	Macro Recall
0	CNN Multi-Class	1	32	(5, 5)	(3, 3)	0.3	1	256	64	11	59183180	0.318043	0.040217	0.026504	0.083333
1	CNN Multi-Class	2	32	(5, 5)	(3, 3)	0.3	1	256	64	11	13129868	0.467890	0.281314	0.358226	0.277578
2	CNN Multi-Class	3	32	(5, 5)	(3, 3)	0.3	1	256	64	10	2750732	0.521407	0.336088	0.412150	0.337448

	Title	Layers	Channels	Kernel	Pool	Dropout	Dense Layers	Dense Units	Batch Size	Epochs Trained	Trainable Params	Accuracy	Macro F1	Macro Precision	Macro Recall
0	CNN Multi-Class	3	32	(7, 7)	(2, 2)	0.4	1	256	64	11	34061324	0.467890	0.287067	0.316775	0.288464
1	CNN Multi-Class	3	32	(7, 7)	(3, 3)	0.4	1	256	64	11	2997260	0.470948	0.246749	0.279161	0.254188
2	CNN Multi-Class	3	32	(7, 7)	(4, 4)	0.4	1	256	64	13	1031180	0.490826	0.291996	0.413798	0.282692

	amecro	amerob	bewwre	bkcchi	daejun	houfin	houspa	rewbla	sonspa	spotow	whcspa
amecro	17	0	0	0	1	0	0	2	1	1	0
amerob	0	36	4	1	1	0	8	3	0	3	1
bewwre	0	0	21	0	2	3	5	3	10	2	1
bkcchi	0	1	1	0	1	0	2	2	3	1	4
daejun	0	0	8	0	25	0	2	1	2	2	1
houfin	1	3	1	0	1	8	11	0	2	1	0
houspa	1	3	4	0	4	0	189	2	1	3	1
norfli	1	4	0	0	1	1	2	0	1	2	0
rewbla	0	16	2	0	1	1	14	15	11	2	0
sonspa	2	10	10	0	3	2	8	3	41	5	3
spotow	2	3	10	0	0	0	5	3	2	20	0
whcspa	0	1	0	0	3	0	2	5	7	1	11